# -*- coding: utf-8 -*-

'''
Created on 2018年9月2日

@author: Dergen Lee

'''

from sklearn import svm

import os
from tensorflow import keras
import gensim
import random
import csv
import gzip
import numpy as np
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt

gnews = 'GoogleNews-vectors-negative300.bin'

unzipped = os.path.join(os.environ['GENSIM_DATASETS'], 'GoogleNews', gnews)

print('Getting pretrained model: GoogleNews-vectors-negative300.bin')
if not os.path.isfile(unzipped):
    path = keras.utils.get_file(os.path.join(os.environ['GENSIM_DATASETS'], 'GoogleNews', gnews + '.gz'),
              'https://s3.amazonaws.com/dl4j-distribution/%s.gz' % gnews)
    with open(unzipped, 'wb') as fout:
        g = gzip.GzipFile(mode="rb", fileobj=open(path, 'rb'))
        fout.write(g.read())

print('Loading the pretrained word embedding model')
model = gensim.models.KeyedVectors.load_word2vec_format(unzipped, binary=True)

print('Most similar of Germany:')
print(model.most_similar(positive=['Germany']))

countries = list(csv.DictReader(open('E:/ai.projects/deep_learning_cookbook/data/countries.csv')))
positive = [x['name'] for x in random.sample(countries, 80)]

negative = random.sample(model.vocab.keys(), 5000)
print(negative[:4])

labelled = [(p, 1) for p in positive] + [(n, 0) for n in negative]
random.shuffle(labelled)
X = np.asarray([model[w] for w, l in labelled])
y = np.asarray([l for w, l in labelled])

print('Svm...')
TRAINING_FRACTION = 0.7
cut_off = int(TRAINING_FRACTION * len(labelled))
clf = svm.SVC(kernel='linear')
clf.fit(X[:cut_off], y[:cut_off])

res = clf.predict(X[cut_off:])

missed = [country for (pred, truth, country) in
          zip(res, y[cut_off:], labelled[cut_off:]) if pred != truth]

print(100 - 100 * float(len(missed)) / len(res), missed)

all_predictions = clf.predict(model.syn0)

res = []
for word, pred in zip(model.index2word, all_predictions):
    if pred:
        res.append(word)
    if len(res) == 150:
        break

print(res)